/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2008-12-15 
 * LastModified: 2008-12-15
 */
package edu.hit.pact.pgse.crawler.download;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.SimpleTimeZone;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HeaderElement;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.htmlparser.util.ParserException;

import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.crawler.util.Anchor;
import edu.hit.pact.pgse.crawler.util.CustomHttpClient;
import edu.hit.pact.pgse.crawler.util.CustonLinkBean;
import edu.hit.pact.pgse.util.Globals;
import edu.hit.pact.pgse.util.Utilities;

/**
 * @author Meteor
 *
 */
public class WebPageFetcher extends AbstractFetcher{
	
	private Log logger;
	
	private CustonLinkBean linkBean;
	
	
	public WebPageFetcher(CustomHttpClient client, String defalutCharSet,
			int readTimeout, int maxContentLength) {
		this(LogFactory.getLog(WebPageFetcher.class), client, defalutCharSet,
				readTimeout, maxContentLength,
				DEFAULT_USER_AGENT);
	}
		
	public WebPageFetcher(CustomHttpClient client, String defalutCharSet,
			int readTimeout, int maxContentLength, String userAgent) {
		this(LogFactory.getLog(WebPageFetcher.class), client, defalutCharSet,
				readTimeout, maxContentLength, userAgent);
	}
	
	public WebPageFetcher(Log logger, CustomHttpClient client,
			String defalutCharSet, int readTimeout, int maxContentLength,
			String userAgent) {
		super(client, defalutCharSet, readTimeout, maxContentLength, userAgent);
		
		this.logger = logger;
		
		this.linkBean = new CustonLinkBean();
	}
	
	public RawWebPage fetchURL(URL url, int type, String belongTo, String channel, String place) {
		return fetchURL(null, url, type, belongTo, channel, place);
	}
	
	public RawWebPage fetchURL(String refereUrl, URL url, int type, String belongTo, String channel, String place) {
		return fetchURL(refereUrl, url, null, type, belongTo, channel, place);
	}
	
	public RawWebPage fetchURL(URL url, RawWebPage property, int type, String belongTo, String channel, String place) {
		return fetchURL(null, url, property, type, belongTo, channel, place);
	}
	

	
	public boolean checkUpdate(URL url,String lastModified,int type){
		HttpMethod method = new GetMethod(url.toString());
		setRequestHeader(method, null, url.getHost(), lastModified, null);
		setParameter(method);
		
		try {
			int statusCode = client.executeMethod(method);
			if (statusCode == HttpStatus.SC_NOT_MODIFIED){
				return false;
			}else{
				return true;
			}
			
		} catch (HttpException e) {
			
			e.printStackTrace();
		} catch (IOException e) {
			
			e.printStackTrace();
		}finally{
			method.releaseConnection();
		}
		
		return true;
	}
	
	public RawWebPage fetchURL(String refereUrl, URL url, RawWebPage property, int type, String belongTo, String channel, String place) {
		HttpMethod method = new GetMethod(url.toString());
		
		if (property == null) {
			setRequestHeader(method, refereUrl, url.getHost(), null, null);
		} else {
			setRequestHeader(method, refereUrl, url.getHost(), property.getLastModified(), null);
		}//end if
		
		setParameter(method);
		
		try {
			int statusCode = client.executeMethod(method);
			
			/*
			 * fetch page with If-Modified-Since header and If-None-Match header.
			 * if the page does't change, the http server will return SC_NOT_MODIFIED(304) statusCode
			 */
//			if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
//				logger.info(url  + " not modified");
//				property.setFromCache(true);
//				return property;
//			}//end if
			
			/*
			 * Handling redirects manually
			 * 
			 * 301 Moved Permanently. HttpStatus.SC_MOVED_PERMANENTLY
			 * 302 Moved Temporarily. HttpStatus.SC_MOVED_TEMPORARILY
			 * 303 See Other. HttpStatus.SC_SEE_OTHER
			 * 307 Temporary Redirect. HttpStatus.SC_TEMPORARY_REDIRECT
			 */
			if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY
					||  statusCode == HttpStatus.SC_MOVED_TEMPORARILY 
					||  statusCode == HttpStatus.SC_SEE_OTHER
					||  statusCode == HttpStatus.SC_TEMPORARY_REDIRECT) {
				Header locationHeader = method.getResponseHeader("location");		
				
				if (locationHeader != null) {
		            String redirectLocation = locationHeader.getValue();
		            
		            String pageRawContent = "redirect to " + redirectLocation;
					String charset = defaultCharSet;
					return constructWebPageProperty(url, pageRawContent, charset, method, type, belongTo, channel, place);
		        } else {
		            // The response is invalid and did not provide the new location for
		            // the resource.  Report an error or possibly handle the response
		            // like a 404 Not Found error.
		        	statusCode = 404;
		        	return null;
		        }//end if
			} //end if
			
			/*
			 * http server return 200 statusCode.
			 */
			if (statusCode == HttpStatus.SC_OK){
				InputStream inputStream = method.getResponseBodyAsStream();
				byte content[] = Utilities.getBytes(inputStream, maxContentLength);
				
				String pageRawContent = new String(content, defaultCharSet);
				String charset = getCharset(method, pageRawContent);
				if (null != charset && 0 != charset.length() && !charset.equalsIgnoreCase(defaultCharSet)) {
					pageRawContent = new String(content, charset);
				}//end if	.
//			    String last =method.getResponseHeader("last-modified").getValue();
//			    if(last != null){
//			        System.out.println("lastModified in web page fetchor : " + last);
//			    }else {
//			    	System.out.println("lastmodified is null");
//			    	return null;
//			    }
			    
			    
				return constructWebPageProperty(url, pageRawContent, charset, method, type, belongTo, channel, place);
			}//end if
			
		} catch (Exception e) {
			logger.error("get " + url + " failed: "+ e.getMessage(), e);
		} finally {
			// be sure the connection is released back to the connection manager
			method.releaseConnection();
		}//end try
		
		logger.warn("get " + url + " failed: "+ method.getStatusLine());
		return null;
	}
	
	public String fetchNormalFile(URL url) {
		HttpMethod method = new GetMethod(url.toString());
		
		setRequestHeader(method, null, url.getHost(), null, null);
		setParameter(method);
		
		String fileContent = null;
		try {
			int statusCode = client.executeMethod(method);
			
			if (statusCode == HttpStatus.SC_OK) {
				InputStream inputStream = method.getResponseBodyAsStream();
				byte content[] = Utilities.getBytes(inputStream, maxContentLength);
				
				fileContent = new String(content, defaultCharSet);
			}//end if
		} catch (Exception e) {
			logger.error("get " + url + " failed: "+ e.getMessage(), e);
		} finally {
			// be sure the connection is released back to the connection manager
			method.releaseConnection();
		}//end try
		
		return fileContent;
	}
	
	private RawWebPage constructWebPageProperty(URL url, String pageRawContent, String charset, HttpMethod method, int type, 
			String belongTo, String channel, String place) {
		RawWebPage property = new RawWebPage(url.toString());
//		logger.error("******"+belongTo+channel+place);
		property.setBelongTo(belongTo);
		property.setChannel(channel);
		property.setPlace(place);
		
		property.setPageRawContent(pageRawContent);
		property.setCharSet(charset);
		property.setLastModified(getLastModified(method));
		property.setType(type);
		
		/*
		 * extract sub link
		 */
		HashMap<String,URL> linksAnchor = new HashMap<String,URL>();
		ArrayList<Anchor> anchorList = new ArrayList<Anchor>();
		List<URL> subLinks = this.linkBean.extractLink(url, pageRawContent, charset,linksAnchor,anchorList);
		String text = null;
		try {
			text = this.linkBean.extractText(url, pageRawContent, charset);
		} catch (ParserException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}		
		/*
		 * filter external links
		 */
		ArrayList<String> interLinks = filterExternalLink(subLinks);
		
		property.setCommentLinks(interLinks);
		property.setLinksAnchor(linksAnchor);
		property.setAnchorList(anchorList);
		property.setText(text);
		return property;
	}
	
	private ArrayList<String> filterExternalLink(List<URL> urls) {
		ArrayList<String> internalLinks = new ArrayList<String>();

		for (URL url : urls) {
			internalLinks.add(url.toString());
		}//end for
		
		return internalLinks.isEmpty() ? null : internalLinks;
	}
	
	private String getLastModified(HttpMethod method) {
		Header lastModifiedHeader = method.getResponseHeader("last-modified");
		
		if (lastModifiedHeader != null) {
			return lastModifiedHeader.getValue();
		} else {
			DateFormat df = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US);
			Calendar cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"));
			df.setCalendar(cal);
			return df.format(new Date());
		}//end if
	}
	
	private String getCharset(HttpMethod method, String pageContent) {
		String charset = null;
		
		/*
		 * fetch charset from responseHeader
		 */
		Header header = method.getResponseHeader("Content-Type");
		if (header != null) {
			HeaderElement values[] = header.getElements();
			
			// I expect only one header element to be there   
			// No more. no less
			if (values.length == 1) {
				NameValuePair param = values[0].getParameterByName("charset"); 
				
				if (param != null) {   
					// If I get anything "funny"    
					// UnsupportedEncondingException will result   
					charset = param.getValue();   
				}    
			}
		}
		
		/*
		 * fetch meta string
		 * such as <meta http-equiv="Content-Type" content="text/html; charset=gb2312" />
		 */
		int prePos = pageContent.indexOf("<meta http-equiv=");
		if (prePos == -1){
			return charset;
		}
		
		int postPos = pageContent.indexOf(">", prePos);
		if (postPos == -1){
			return charset;
		}

		String metaStr = pageContent.substring(prePos, postPos);
		
		/*
		 * fetch charset
		 */
		prePos = metaStr.indexOf("charset=");
		if (prePos == -1){
			return charset;
		}
		
		postPos = metaStr.indexOf("\"", prePos);
		if (postPos == -1){
			return charset;
		}
		
		return metaStr.substring(prePos + 8, postPos);
	}
		
}
